# NO USE, please do NOT use it
# Updated by Wei on 20140915
exit(1)

from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import os
import random
import sys
import time
from sets import Set
from random import randint
import re
import numpy as np
from os import walk

ofn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/trainingFile_20140915_step8_extended.arff"
ofh = open(ofn,"w")

unprunedQueryTermsSamplingValuesDict = {}
ifn3 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/Unpruned_100%_queryTermsSamplingValues_20140912"
ifh = open(ifn3,"r")
l = ifh.readline()
while l:
	le = l.strip().split(" ")
	contentLine = ""
	term = le[0]
	for value in le[1:]:
		contentLine += value + ","
	unprunedQueryTermsSamplingValuesDict[term] = contentLine[:-1]
	l = ifh.readline()
ifh.close()
print "Overall:"
print "len(unprunedQueryTermsSamplingValuesDict):",len(unprunedQueryTermsSamplingValuesDict)

upp5_1PercentQueryTermsSamplingValuesDict = {}
ifn4 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/UPP-5_1%_queryTermsSamplingValues_20140912"
ifh = open(ifn4,"r")
l = ifh.readline()
while l:
	le = l.strip().split(" ")
	contentLine = ""
	term = le[0]
	for value in le[1:]:
		contentLine += value + ","
	upp5_1PercentQueryTermsSamplingValuesDict[term] = contentLine[:-1]
	l = ifh.readline()	
ifh.close()
print "Overall:"
print "len(upp5_1PercentQueryTermsSamplingValuesDict):",len(upp5_1PercentQueryTermsSamplingValuesDict)
print "upp5_1PercentQueryTermsSamplingValuesDict['zydeco']:",upp5_1PercentQueryTermsSamplingValuesDict['zydeco']

qidWithTermListDict = {}
ifn1 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/Unpruned_OR_100%_auxFile_20140913"
ifh = open(ifn1,"r")
l = ifh.readline()
while l:
    if l.strip().startswith("qid"):
        qid = l.strip().split(" ")[1]
        print qid
        termList = []
        elements = ifh.readline().strip().split(" ")
        for term in elements[1:]:
            termList.append(term)
        qidWithTermListDict[qid] = termList
    l = ifh.readline()
print "len(qidWithTermListDict):",len(qidWithTermListDict)
print "sth:",qidWithTermListDict["14104"]
ifh.close()

ifn2 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/trainingFile_20140913_step8.arff"
ifh = open(ifn2,"r")
print "ifn2:",ifn2
l = ifh.readline()
while l:
    le = l.strip().split(",")
    # print "len(le):",len(le)
    if len(le) == 37:
    	# print "in1"
    	outputLine = ""
    	qid = le[0]
    	part1 = ""
    	part2 = ""
        part3 = ""
        for i in range(0,2):
        	part1 = le[i] + ","
        	print "part1:",part1
        outputLine += part1
        for i in range(2,12):
        	part2 += le[i] + ","
        	print "part2:",part2
        outputLine += part2
        '''
        for term in qidWithTermListDict[qid]:
        	if term not in upp5_1PercentQueryTermsSamplingValuesDict:
        		part3 += "0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0" + ","
        	else:
        		part3 += upp5_1PercentQueryTermsSamplingValuesDict[term] + ","
        print "part3:",part3
        outputLine += part3
        for i in range(0,10-len(qidWithTermListDict)):
			outputLine += "0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0" + ","
		'''
		for i in range(12,22):
			outputLine += le[i] + ","
		for term in qidWithTermListDict[qid]:
			if term not in unprunedQueryTermsSamplingValuesDict:
		 		outputLine += "0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0" + ","
	 		else:
	 			outputLine += unprunedQueryTermsSamplingValuesDict[term] + ","
		'''
		for i in range(0,10-len(qidWithTermListDict)):
			outputLine += "0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0" + ","
		'''
        for i in range(22,32):
        	outputLine += le[i] + ","
		
		for i in range(32,37):	
			outputLine += le[i] + ","
		
		print "outputLine4:",outputLine
		ofh.write(outputLine + "\n")
    else:
        ofh.write(l)

    l = ifh.readline()
ifh.close()
ofh.close()
print "Overall:"
print "ifn1:",ifn1
print "ifn2:",ifn2
print "ofn:",ofn
